#!/usr/bin/python
#coding=utf-8

import sys
import os
import rados
import threading
import commands
import time
import copy
import json
import uuid
import logging
import logging.config

import global_list
from command import *

CONFPATH="/home/dongrui/liushuai/cephtest/mycluster"

class ReqDisk:
    #self.h=链接集群的handle
    def __init__(self,handle,dict,reqid,logger):
        self.h=handle
        self.act=dict['act']
        self.cat=dict['cat']
        self.host=dict['host']
        self.sn=dict['sn']
        self.item=dict['item']
        self.reqid=int(reqid)
        self.logger=logger

    def ThreadDealer(self):
        resp={}
        resp['act']=str(self.act)
        resp['type']='disk'
        resp['status']='running'
        if self.act == 'getinfo':
            resp['cat']=self.cat
        if self.act == 'replacedata' or self.act == 'replacejournal':
            resp['step']='0'
            resp['needreplace']='false'
            resp['sn']='0'

        #tname=threading.currentThread().getName()
        mtx=global_list.get_value('MUTEX')
        #加锁->获取结果列表->追加结果->更新列表->解锁
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

        #执行具体命令
        if self.act == 'getinfo':
            if self.cat == 'data':
                ret=self.getinfo_by_data()
            else:
                ret=self.getinfo_by_journal()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'checkdisk':
            ret,status=self.checkdisk(self.sn)
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'replacedata':
            ret=self.replacedata()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'replacejournal':
            ret=self.replacejournal()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        else:
            pass
 
        #更新结果
        mtx=global_list.get_value('MUTEX')
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

    #得到主机列表
    def gethosts(self):
        respdict=dict()

        cmd="/usr/bin/ceph osd tree | /usr/bin/grep -w host | /usr/bin/awk '{print $4}'"
        ret,hosts=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd tree failed,ret is "+str(ret)+" ,output is "+str(hosts))
            respdict['err']='ceph osd tree failed'
            return respdict
        lines=hosts.split('\n')
        respdict['hosts']=lines

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict
        
    #查询物理机上的OSD情况/查找某个OSD对应的日志盘信息
    def getinfo_by_data(self):
        respdict=dict()
        resplist=[]

        respdict['host']=self.host
        osdtree = CephClusterCommand(self.h, prefix='osd tree', format='json')
        if osdtree.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd tree failed, error is "+osdtree['err'])
            respdict['err']=osdtree['err']
            return respdict
        result=osdtree['result']
        #查找主机信息
        host=[ x for x in result['nodes'] if x['type']=='host' and x['name']==self.host ]

        #获取该主机上所有OSDs的ID列表
        try:
            children=host[0].get('children')
        except IndexError,e:
            self.logger.error("reqid:"+str(self.reqid)+" get children on "+str(self.host)+" failed, error is "+str(e))
            respdict['err']='get children failed'
            return respdict

        for i in children:
            d=dict()
            d['name']="osd."+str(i)
            path="/var/lib/ceph/osd/ceph-"+str(i)+"/"
            #读取日志盘对应的设备名
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/readlink -f "+str(path)+"journal"
            ret,journal=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" readlink failed, ret is "+str(ret)+" ,output is "+str(journal))
                respdict['err']="readlink journal disk failed"
                return respdict
            d['journal']=journal
            #获取OSD数据盘所在的设备名
            cmd="/usr/bin/ssh "+str(self.host)+" df --output=source "+str(path)+"| grep -vi filesystem"
            ret,device=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" df get source failed, ret is "+str(ret)+" ,output is "+str(device))
                respdict['err']="df data disk failed"
                return respdict
            d['device']=device
            resplist.append(d)

        respdict['osds']=resplist
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #查询物理机上的OSD情况/查找某个OSD对应的日志盘信息
    def getinfo_by_journal(self):
        respdict=dict()
        resplist=[]

        #首先获取journal所在的设备名
        path="/var/lib/ceph/osd/ceph-"+str(self.item)+"/journal"
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/readlink -f "+str(path)
        ret,journal=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" readlink failed, ret is "+str(ret)+" ,output is "+str(journal))
            respdict['err']='readlink journal disk failed'
            return respdict

        #获取PKNAME，也就是分区所在磁盘的名称
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/lsblk -p --output PKNAME "+str(journal)+" | /usr/bin/grep -Ev \"PKNAME|^$\""
        ret,pkname=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" lsblk get pkname failed, ret is "+str(ret)+" ,output is "+str(pkname))
            respdict['err']='lsblk get PKNAME failed'
            return respdict
        
        #最后获取该日志盘的所有分区
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/lsblk -Pp --output=NAME,TYPE,SERIAL "+str(pkname)
        ret,parts=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" lsblk get serail and type failed, ret is "+str(ret)+" ,output is "+str(parts))
            respdict['err']='lsblk get NAME,SERIAL failed'
            return respdict

        allparts=[]
        journal_sn=''
        partslist=parts.strip().split('\n')
        for i in partslist:
            t_name=i.split()[0].split('=')[1].strip('\"')
            t_type=i.split()[1].split('=')[1].strip('\"')
            if t_type == "part":
                allparts.append(t_name)
            if t_type == "disk":
                #获取磁盘SN
                journal_sn=i.split()[2].split('=')[1].strip('\"')
            
        respdict['host']=self.host
        respdict['journal']=pkname
        respdict['serial num']=journal_sn

        #获取该host上所有的OSDs的ID列表
        osdtree = CephClusterCommand(self.h, prefix='osd tree', format='json')
        if osdtree.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd tree failed, error is "+osdtree['err'])
            respdict['err']=osdtree['err']
            return respdict
        result=osdtree['result']
        host=[ x for x in result['nodes'] if x['type']=='host' and x['name']==self.host ]

        children=host[0].get('children')
        osdslist=[]
        for i in children:
            d=dict()
            path="/var/lib/ceph/osd/ceph-"+str(i)+"/"
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/readlink -f "+str(path)+"journal"
            ret,dname=commands.getstatusoutput(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" readlink failed, ret is "+str(ret)+" ,output is "+str(dname))
                respdict['err']='readlink journal disk failed'
                return respdict
            #如果该OSD使用此日志盘
            if dname in allparts:
                osdslist.append('osd.'+str(i))

        respdict['osds']=osdslist
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #辅助函数，获取磁盘SN号
    def get_SN(self,dev):
        respdict=dict()

        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/hdparm -I "+str(dev)+" | /usr/bin/grep \"Serial Number\" | /usr/bin/awk -F':' '{print $2}'"
        ret,val=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" hdparm "+str(dev)+" failed, ret is "+str(ret)+" ,output is "+str(val))
            respdict['err']='get serial number failed'
            return respdict
        else:
            sn=val.strip()
            self.logger.info("reqid:"+str(self.reqid)+" dev: "+str(dev)+" ,serial: "+str(sn))
            respdict['sn']=sn
            return respdict

    #检查磁盘SN号是否正确
    def checkdisk(self,sn):
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/lsblk -p --output NAME,SERIAL | /usr/bin/grep -wi "+str(sn)
        ret,val=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" lsblk get serial failed, ret is "+str(ret)+" ,output is "+str(val))
            return False, 'error'
        else:
            dev=val.strip().split()[0]
            self.logger.info("reqid:"+str(self.reqid)+" dev: "+str(dev)+" ,serial: "+str(sn))
            return True,dev

    #辅助函数，禁止集群触发恢复、回填、均衡操作
    def setno(self):
        ret=os.system('/usr/bin/ceph osd set norecover')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd set norecover failed, ret is "+str(ret))
            return False
        ret=os.system('/usr/bin/ceph osd set nobackfill')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd set nobackfill failed, ret is "+str(ret))
            return False
        ret=os.system('/usr/bin/ceph osd set norebalance')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd set norebalance failed, ret is "+str(ret))
            return False

        self.logger.info("reqid:"+str(self.reqid)+" osd set noreb/norec/noback success")
        return True

    #辅助函数，设置OSD状态为out
    def setout(self):
        cmd='/usr/bin/ceph osd out '+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" set out success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" osd."+str(self.item)+" set out failed, ret is "+str(ret))
            return False

    #辅助函数，登录物理机停止该OSD进程服务
    def stop_osd(self):
        cmd="/usr/bin/systemctl -H "+str(self.host)+" stop ceph-osd@"+str(self.item)+".service"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" stop osd."+str(self.item)+" failed, ret is "+str(ret))
            return False

        cmd="/usr/bin/systemctl -H "+str(self.host)+" is-active ceph-osd@"+str(self.item)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res != 'inactive':
            self.logger.error("reqid:"+str(self.reqid)+" osd."+str(self.item)+" is not inactive, ret is "+str(ret)+",output is "+str(res))
            return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+" is inactive now")
            return True

    #辅助函数，登录物理机停止指定的OSD
    def stop_osd_by_id(self, idx):
        cmd="/usr/bin/systemctl -H "+str(self.host)+" stop ceph-osd@"+str(idx)+".service"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" stop osd."+str(idx)+" failed, ret is "+str(ret))
            return False

        cmd="/usr/bin/systemctl -H "+str(self.host)+" is-active ceph-osd@"+str(idx)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res != 'inactive':
            self.logger.error("reqid:"+str(self.reqid)+" osd."+str(idx)+" is not inactive, ret is "+str(ret)+",output is "+str(res))
            return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(idx)+" is inactive now")
            return True

    #辅助函数，将OSD从集群删除
    def rm_osd_from_cluster(self):
        cmd="/usr/bin/ceph osd rm osd."+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" failed, ret is "+str(ret))
            return False

    #辅助函数，将OSD从集群的CRUSH Map中删除
    def rm_osd_from_crush(self):
        cmd="/usr/bin/ceph osd crush rm osd."+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" crush rm osd."+str(self.item)+" success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" crush rm osd."+str(self.item)+" failed, ret is "+str(ret))
            return False
        
    #辅助函数，将OSD的key从集群删除
    def rm_osd_from_auth(self):
        cmd="/usr/bin/ceph auth del osd."+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" crush rm osd."+str(self.item)+" success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" crush rm osd."+str(self.item)+" failed, ret is "+str(ret))
            return False
    
    #辅助函数，卸载损坏磁盘的挂载目录
    def umount_osd(self):
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/umount /var/lib/ceph/osd/ceph-"+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" umount osd."+str(self.item)+" success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" umount osd."+str(self.item)+" failed, ret is "+str(ret))
            return False

    #辅助函数，删除OSD文件目录
    def rm_osd_dir(self):
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/rm -rf /var/lib/ceph/osd/ceph-"+str(self.item)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" dir success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" dir failed, ret is "+str(ret))
            return False

    #辅助函数，删除OSD文件目录下的journal链接
    def rm_osd_journal(self,idx):
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/rm -rf /var/lib/ceph/osd/ceph-"+str(idx)+"/journal"
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" journal success")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" rm osd."+str(self.item)+" journal failed, ret is "+str(ret))
            return False

    #辅助函数，获取某个OSD对应的journal设备名
    def get_journal(self):
        respdict=dict()

        #获取日志的对应分区
        path="/var/lib/ceph/osd/ceph-"+str(self.item)+"/journal"
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/readlink -f "+str(path)
        ret,journal=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" readlink osd."+str(self.item)+" journal failed, ret is "+str(ret)+",output is "+str(journal))
            respdict['err']='readlink journal failed'
            return respdict
        else:
            respdict['dev']=str(journal)
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+",journal:"+str(journal))
            return respdict

    #辅助函数，获取某个OSD对应的设备名
    def get_data(self):
        respdict=dict()

        #获取OSD的对应分区
        path="/var/lib/ceph/osd/ceph-"+str(self.item)
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/df --output=source "+str(path)+" | /usr/bin/grep -vi filesystem"
        ret,device=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get osd."+str(self.item)+" data failed, ret is "+str(ret)+",output is "+str(device))
            respdict['err']='df osd failed'
            return respdict

        #获取分区所在的磁盘名称
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/lsblk --output=PKNAME "+str(device)+" | /usr/bin/grep -vE \"PKNAME|^$\""
        ret,pkname=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get device "+str(device)+" pkname failed, ret is "+str(ret)+",output is "+str(pkname))
            respdict['err']='lsblk osd failed'
            return respdict
        else:
            self.logger.info("reqid:"+str(self.reqid)+" osd."+str(self.item)+",device: "+str(pkname))
            respdict['dev']=pkname
            return respdict

    #辅助函数，准备新盘，将新盘加入集群，重新分区
    def prepare_osd(self,dev,journal):
        #首先格式化新盘
        cmd="/usr/bin/ceph-deploy disk zap "+str(self.host)+":"+str(dev)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" disk zap failed, ret is "+str(ret))
            return False
      
        #准备新盘
        cmd="/usr/bin/ceph-deploy osd prepare "+str(self.host)+":"+str(dev)+":"+str(journal)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd prepare failed, ret is "+str(ret))
            return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" success")
            return True

    #辅助函数，激活新OSD进程
    def activate_osd(self,dev,journal):
        cmd="/usr/bin/ceph-deploy osd activate "+str(self.host)+":"+str(dev)+"1:"+str(journal)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd activate failed, ret is "+str(ret))
            return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" success")
            return True

    #辅助函数，开启集群恢复、回填、均衡操作
    def unset(self):
        ret=os.system('/usr/bin/ceph osd unset norecover')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" unset norecover failed, ret is "+str(ret))
            return False
        ret=os.system('/usr/bin/ceph osd unset nobackfill')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" unset nobackfill failed, ret is "+str(ret))
            return False
        ret=os.system('/usr/bin/ceph osd unset norebalance')
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" unset norebalance failed, ret is "+str(ret))
            return False
            
        self.logger.info("reqid:"+str(self.reqid)+" osd unset noreb/norec/noback success")
        return True

    #辅助函数，检查集群PG状态是否为全部为active+clean
    def get_healthOK(self):
        pg_stat = CephClusterCommand(self.h, prefix='pg stat', format='json')
        if pg_stat.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" pg stat failed, error is "+pg_stat['err'])
            return False
        result=pg_stat['result']
        num_pgs=result.get('num_pgs')
        state=result.get('num_pg_by_state')
        num_act_cln_pgs=0
        for i in state:
            if i['name'] == "active+clean":
                num_act_cln_pgs=i['num']
        if num_pgs == num_act_cln_pgs:
            self.logger.info("reqid:"+str(self.reqid)+" all PGs are active+clean")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" not all PGs are active+clean")
            return False
 
    #辅助函数，更新step
    def update_step(self,step,flag):
        mtx=global_list.get_value('MUTEX')
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        temp=res[int(self.reqid)]
        temp['step']=step
        temp['needreplace']=flag
        res[int(self.reqid)]=temp
        global_list.set_value('RESULTS',res)
        mtx.release()

    #实施更换数据盘操作
    def replacedata(self):
        LOC_DIFF=True

        #获取data和journal所在磁盘名称
        ret=self.get_journal()
        if True == ret.has_key('err'):
            self.update_step('1-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" get journal failed")
            return False
        journal=ret['dev']
        self.update_step('1-1','false')
        
        ret=self.get_data()
        if True == ret.has_key('err'):
            self.update_step('1-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" get data failed")
            return False
        data=ret['dev']
        self.update_step('1-1','false')

        #data和journal在同一磁盘
        if journal == data:
            LOC_DIFF=False
            self.logger.info("reqid:"+str(self.reqid)+" data and journal are on the same disk")
        else:
            LOC_DIFF=True
            self.logger.info("reqid:"+str(self.reqid)+" data and journal are on the diff disk")

        #禁止集群自动恢复/回填/平衡
        ret=self.setno()
        if ret == False:
            self.update_step('1-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" set noreb/norec/noback failed")
            return False
        self.update_step('1-1','false')

        #设置OSD为out
        ret=self.setout()
        if ret == False:
            self.update_step('1-2','false')
            self.logger.error("reqid:"+str(self.reqid)+" set osd out failed")
            return False
        self.update_step('1-2','false')

        #登录主机停止指定OSD
        ret=self.stop_osd()
        if ret == False:
            self.update_step('1-3','false')
            self.logger.error("reqid:"+str(self.reqid)+" stop osd failed")
            return False
        self.update_step('1-3','false')

        #从集群删除故障OSD
        ret=self.rm_osd_from_cluster()
        if ret == False:
            self.update_step('1-4','false')
            self.logger.error("reqid:"+str(self.reqid)+" rm osd failed")
            return False
        self.update_step('1-4','false')

        #从crush中删除故障OSD
        ret=self.rm_osd_from_crush()
        if ret == False:
            self.update_step('1-5','false')
            self.logger.error("reqid:"+str(self.reqid)+" rm osd from crush failed")
            return False
        self.update_step('1-5','false')

        #删除故障OSD的认证信息
        ret=self.rm_osd_from_auth()
        if ret == False:
            self.update_step('1-6','false')
            self.logger.error("reqid:"+str(self.reqid)+" rm osd from auth failed")
            return False
        self.update_step('1-6','false')

        #卸载OSD所在磁盘
        ret=self.umount_osd()
        if ret == False:
            self.update_step('1-7','false')
            self.logger.error("reqid:"+str(self.reqid)+" umount osd failed")
            return False
        self.update_step('1-7','false')

        #删除OSD磁盘
        ret=self.rm_osd_dir()
        if ret == False:
            self.update_step('1-8','false')
            self.logger.error("reqid:"+str(self.reqid)+" rm osd dir failed")
            return False
        self.update_step('1-8','true')

        #等待换盘
        disk_sn='0'
        while True:
            mtx=global_list.get_value('MUTEX')
            mtx.acquire()
            res=global_list.get_value('RESULTS')
            ret=res[int(self.reqid)]
            if ret['sn'] != '0':
                disk_sn=copy.deepcopy(ret['sn'])
            global_list.set_value('RESULTS',res)
            mtx.release()
            if disk_sn != '0':
                self.logger.info("reqid:"+str(self.reqid)+" get new disk serial number: "+str(disk_sn))
                break
            else:
                self.logger.info("reqid:"+str(self.reqid)+" waiting new disk serial number...")
                time.sleep(60)

        #检查新插入磁盘的SN
        ret,dev=self.checkdisk(disk_sn)
        if ret == False:
            self.update_step('3-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" check new disk serial number failed")
            return False
        self.update_step('3-1','false')

        #ceph-deploy add
        #进入xx目录，拷贝conf，执行添加mon命令
        os.chdir(CONFPATH)
        cmd="/usr/bin/cp -p /etc/ceph/ceph.conf "+str(CONFPATH)
        ret=os.system(cmd)
        if ret != 0:
            self.update_step('3-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf to "+str(CONFPATH)+" failed,ret is "+str(ret))
            return False
        self.update_step('3-1','false')

        #准备OSD
        ret=self.prepare_osd(dev,journal)
        if ret == False:  
            self.update_step('3-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" prepare osd failed")
            return False
        self.update_step('3-1','false')
        
        #激活OSDelre_osd
        ret=self.activate_osd(dev,journal)
        if ret == False:
            self.logger.error("reqid:"+str(self.reqid)+" activate osd failed")
            #return False
        self.update_step('3-2','false')

        #解除禁止集群恢复/回填/平衡
        ret=self.unset()
        if ret == False:
            self.update_step('3-3','false')
            self.logger.error("reqid:"+str(self.reqid)+" unset noreb/norec/noback failed")
            return False 
        self.update_step('3-3','false')

        #等待集群回复正常
        while True:
            ret=self.get_healthOK()
            if ret == True:
                self.update_step('3-4','false')
                break
            else:
                time.sleep(5)
                continue

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return True

    #辅助函数，下刷journal数据到osd
    def flush_journal(self, idx):
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/ceph-osd -i "+str(idx)+" --flush-journal"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" osd."+str(idx)+" flush-journal failed,ret is "+str(ret))
            return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" success")
            return True

    #辅助函数，根据数据盘个数，对journal磁盘分区
    def make_journal(self, journal, num):
        #首先格式化磁盘
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/sbin/sgdisk --zap-all "+str(journal)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" sgdisk zap journal "+str(journal)+" failed,ret is "+str(ret))
            return False

        #获取磁盘大小和单位
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/sbin/sgdisk -p "+str(journal)+" | /usr/bin/grep Disk | /usr/bin/grep sectors | /usr/bin/awk \'{print $5,$6}\'"
        ret,buf=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get journal size failed, ret is "+str(ret)+",output is "+str(buf))
            return False
        self.logger.info("reqid:"+str(self.reqid)+" host: "+str(self.host)+",journal: "+str(journal)+",size: "+str(buf))

        #得到大小和单位
        size=float(buf.split()[0])
        unit=buf.split()[1]
        if unit == 'MiB':
            #以下以MiB作为基本单位
            pass
        elif unit == 'GiB':
            size=size*1000
        elif unit == 'TiB':
            size=size*1000000
        else:
            self.logger.error("reqid:"+str(self.reqid)+" unsupport unit: "+str(unit))
            return False

        #计算单个分区的大小，单位为MiB
        partsize=int(size)/num

        #利用sgdisk工具进行分区
        for i in range(num):
            idx=i+1
            #生成UUID
            name=str(time.time())
            disk_uuid=uuid.uuid5(uuid.NAMESPACE_DNS,name)
            cmd="/usr/bin/ssh "+str(self.host)+" \'/usr/sbin/sgdisk --new="+str(idx)+":0:+"+str(partsize)+"M: --change-name="+str(idx)+":\"ceph journal\" --partition-guid="+str(idx)+":"+str(disk_uuid)+" --typecode="+str(idx)+":45b0969e-9b03-4f30-b4c6-b4b80ceff106 --mbrtogpt -- "+str(journal)+"\'"
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" format journal "+str(journal)+" failed,ret is "+str(ret))
                return False
            time.sleep(1)

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return True

    #将新建分区链接到OSD的journal
    def make_link(self, journal, list):
        partnum=1

        #list=与该journal关联的OSDs列表
        for i in list:
            #获取OSD的ID
            idx=int(i.split('.')[1])
            #日志盘为nvme类型
            if journal[5:9]=='nvme':
                partname=journal+'p'+str(partnum)
            #日志盘为s-ata类型
            elif journal[5:7]=='sd':
                partname=journal+str(partnum)
            else:
                self.logger.error("reqid:"+str(self.reqid)+" unsupport disktype: "+str(journal))
                return False
            #获取PARTUUID
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/sbin/blkid -s PARTUUID "+partname+" -o udev"
            ret,buf=commands.getstatusoutput(cmd)
            if ret != 0 or buf == "":
                self.logger.error("reqid:"+str(self.reqid)+" get journal partuuid failed, ret is "+str(ret)+",output is "+str(buf))
                return False
            partuuid=buf.split('=')[1]
            elf.logger.info("reqid:"+str(self.reqid)+" partname: "+str(partname)+",partuuid: "+str(partuuid))
 
            #创建软连接
            src="/dev/disk/by-partuuid/"+str(partuuid)
            dst="/var/lib/ceph/osd/ceph-"+str(idx)+"/journal"
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/ln -s "+str(src)+" "+str(dst)
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" ln "+str(src)+" to "+str(dst)+" failed, ret is "+str(ret))
                return False

            #更改权限
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/chown ceph:ceph "+str(src)
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" chown failed, ret is "+str(ret))
                return False
            partnum+=1

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return True

    #实施更换日志盘操作
    def replacejournal(self):
        #禁止集群自动恢复/回填/平衡
        ret=self.setno()
        if ret == False:
            self.update_step('1-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" set noreb/norec/noback failed")
            return False
        self.update_step('1-1','false')

        #获取与该journal相关的所有OSDs列表
        ret=self.getinfo_by_journal()
        if True == ret.has_key('err'):
            self.update_step('1-2','false')
            self.logger.error("reqid:"+str(self.reqid)+" get journal info failed")
            return False
        self.update_step('1-2','false')

        journal_osds=ret['osds']
        journal=ret['journal']
        self.logger.info("reqid:"+str(self.reqid)+" host: "+str(self.host)+",journal: "+str(journal)+",osds: "+str(journal_osds))

        #停止所有跟journal关联的OSD进程
        for i in journal_osds:
            ret = self.stop_osd_by_id(i.split('.')[1])
            if False == ret:
                self.update_step('1-3','false')
                self.logger.error("reqid:"+str(self.reqid)+" stop osd failed")
                return False
        self.update_step('1-3','false')
        
        #flush journal
        for i in journal_osds:
            ret = self.flush_journal(i.split('.')[1])
            if False == ret:
                self.update_step('1-4','false')       
                self.logger.error("reqid:"+str(self.reqid)+" flush osd journal failed")
                return False
        self.update_step('1-4','true')       

        #等待换盘
        disk_sn='0'
        while True:
            mtx=global_list.get_value('MUTEX')
            mtx.acquire()
            res=global_list.get_value('RESULTS')
            ret=res[int(self.reqid)]
            if ret['sn'] != '0':
                disk_sn=copy.deepcopy(ret['sn'])
            global_list.set_value('RESULTS',res)
            mtx.release()
            if disk_sn != '0':
                self.logger.info("reqid:"+str(self.reqid)+" get new journal disk serial number: "+str(disk_sn))
                break
            else:
                self.logger.info("reqid:"+str(self.reqid)+" waiting new journal disk serial number...")
                time.sleep(60)

        #给新盘分区
        ret=self.make_journal(journal, len(journal_osds))
        if False == ret:
            self.update_step('3-1','false')
            self.logger.error("reqid:"+str(self.reqid)+" make osd journal failed")
            return False
        self.update_step('3-1','false')

        #删除关联OSDs目录下的journal链接
        for i in journal_osds:
            ret = self.rm_osd_journal(i.split('.')[1])
            if False == ret:
                self.update_step('3-2','false')
                self.logger.error("reqid:"+str(self.reqid)+" rm osd journal failed")
                return False
        self.update_step('3-2','false')

        #重新建立关联OSDs的journal链接
        ret = self.make_link(journal, journal_osds)
        if False == ret:
            self.update_step('3-3','false')
            self.logger.error("reqid:"+str(self.reqid)+" make osd journal link failed")
            return False
        self.update_step('3-3','false')

        #利用ceph-osd重新刷新journal
        for i in journal_osds:
            cmd="/usr/bin/ssh "+str(self.host)+" /usr/bin/ceph-osd --cluster ceph -i "+str(i.split('.')[1])+" --mkjournal"
            ret=os.system(cmd)
            if ret != 0:
                self.update_step('3-5','false')
                self.logger.error("reqid:"+str(self.reqid)+" osd mkjournal failed, ret is "+str(ret))
                return False
        self.update_step('3-5','false')
   
        #远程启动OSDs
        for i in journal_osds:
            cmd="/usr/bin/systemctl -H "+str(self.host)+" start ceph-osd@"+str(i.split('.')[1])
            ret=os.system(cmd)
            if ret != 0:
                self.update_step('3-6','false')
                self.logger.error("reqid:"+str(self.reqid)+" start osd."+str(i.split('.')[1])+" failed, ret is "+str(ret))
                return False
        self.update_step('3-6','false')

        #解除禁止恢复/回填/均衡
        ret=self.unset()
        if ret == False:
            self.update_step('3-7','false')
            self.logger.error("reqid:"+str(self.reqid)+" osd unset noreb/norec/noback failed")
            return False
        self.update_step('3-7','false')

        #等待集群回复正常
        while True:
            ret=self.get_healthOK()
            if ret == True:
                self.update_step('3-8','false')
                break
            else:
                time.sleep(5)
                continue

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return True

    #析构
    def __del__(self):
        pass
